import scrapy
from scrapy import FormRequest
from scrapy.http import JsonRequest
from scrapy import Spider
import json 
import re 

'''
爬取 Ultimate Guitar 网站的歌手信息
'''
class UltimateGuitarSpider(scrapy.Spider):
    name = 'ug-spider'
    loginUrl = 'https://www.ultimate-guitar.com/user/auth/processSignIn'
    start_urls = ['https://www.ultimate-guitar.com/explore?type[]=Chords']

    custom_settings = {
        'ITEM_PIPELINES': {
            'pipelines.ugpipeline.UltimateGuitarPipeline': 1,
            'pipelines.ugpipeline.ArtistPipeline': 2,
        }
    }

    def parse(self, response):
        # Step 1: 提取 CSRF Token（不同网站位置可能不同）
        # csrf_token = response.css('input[name=csrf_token]::attr(value)').get()
        
        jsstore = response.xpath('//div[@class="js-store"]').xpath('//div/@data-content').extract()[0]
        jsstore = json.loads(jsstore)
        yield jsstore
        # print(type(jsstore)
        # with open('data.json', 'w', encoding='utf-8') as f:
        #     f.write(jsstore)
